From 97dca28dde325ceb0cb91041fdf013e5bf35e708 Mon Sep 17 00:00:00 2001 From: "Karl O. Pinc" Date: Tue, 5 Jan 2021 15:17:14 -0600 Subject: [PATCH] Fork from pgwui_upload --- .coveragerc | 2 +- .gitignore | 2 +- MANIFEST.in | 4 +- Makefile | 2 +- setup.py | 17 +- src/pgwui_upload/templates/upload.mak | 116 ------------ .../VERSION | 0 .../__init__.py | 2 +- .../check_settings.py | 22 +-- .../exceptions.py | 16 +- .../pgwui_upload_core.py} | 22 +-- src/pgwui_upload_core/templates/upload.mak | 82 +++++++++ .../views/__init__.py | 0 .../views/upload.py | 165 +++++++----------- tests/test_check_settings.py | 83 +++------ tests/test_pgwui_upload.py | 98 ----------- tests/test_pgwui_upload_core.py | 55 ++++++ tests/views/test_upload.py | 147 ++++++++-------- tox.ini | 6 +- 19 files changed, 336 insertions(+), 505 deletions(-) delete mode 100644 src/pgwui_upload/templates/upload.mak rename src/{pgwui_upload => pgwui_upload_core}/VERSION (100%) rename src/{pgwui_upload => pgwui_upload_core}/__init__.py (93%) rename src/{pgwui_upload => pgwui_upload_core}/check_settings.py (72%) rename src/{pgwui_upload => pgwui_upload_core}/exceptions.py (85%) rename src/{pgwui_upload/pgwui_upload.py => pgwui_upload_core/pgwui_upload_core.py} (55%) create mode 100644 src/pgwui_upload_core/templates/upload.mak rename src/{pgwui_upload => pgwui_upload_core}/views/__init__.py (100%) rename src/{pgwui_upload => pgwui_upload_core}/views/upload.py (64%) delete mode 100644 tests/test_pgwui_upload.py create mode 100644 tests/test_pgwui_upload_core.py diff --git a/.coveragerc b/.coveragerc index 01602c9..0fffbe0 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,7 @@ [run] branch = True source = - pgwui_upload + pgwui_upload_core views [report] diff --git a/.gitignore b/.gitignore index aae03e1..c1cbb75 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ devel/ dist/ docs/build/ README.html -src/pgwui_upload.egg-info/ +src/pgwui_upload_core.egg-info/ diff --git a/MANIFEST.in b/MANIFEST.in index c71abd9..293697a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,6 @@ include *.mk include .coveragerc include LICENSE.txt include Makefile -include src/pgwui_upload/VERSION -include src/pgwui_upload/templates/*.mak +include src/pgwui_upload_core/VERSION +include src/pgwui_upload_core/templates/*.mak include tox.ini diff --git a/Makefile b/Makefile index 92ee0d6..aef7175 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Copyright (C) 2016, 2017, 2018, 2019 The Meme Factory, Inc. # http://www.karlpinc.com/ -# This file is part of PGWUI_Upload. +# This file is part of PGWUI_Upload_Core. # # This program is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License diff --git a/setup.py b/setup.py index 8d66212..ca18ebd 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ def filter_readme(): here = path.abspath(path.dirname(__file__)) # Get program version -with open(path.join(here, 'src', 'pgwui_upload', 'VERSION'), +with open(path.join(here, 'src', 'pgwui_upload_core', 'VERSION'), encoding='utf-8') as version_file: version = version_file.read().strip() @@ -66,7 +66,7 @@ tests_require = [ ] setup( - name='pgwui_upload', + name='pgwui_upload_core', # Versioning is major.minor.fixes. Major releases change (after 1.0.0) # when backward incompatibility is introduced. Minor releases introduce @@ -74,16 +74,16 @@ setup( version=version, description=( - 'A web interface for bulk PostgreSQL data validation and upload.'), + 'PGWUI API for bulk PostgreSQL data validation and upload.'), long_description=long_description, long_description_content_type='text/x-rst', # The project's main homepage. - url='http://pgwui_upload.readthedocs.io/', + url='http://pgwui_upload_core.readthedocs.io/', # Author details author='Karl O. Pinc', - author_email='kop@meme.com', + author_email='kop@karlpinc.com', # Choose your license license='AGPLv3+', @@ -156,6 +156,7 @@ setup( # Run-time dependencies. install_requires=[ + 'attrs', 'markupsafe', 'pgwui_common', 'psycopg2', @@ -174,7 +175,7 @@ setup( # installed, specify them here. If using Python 2.6 or less, then these # have to be included in MANIFEST.in as well. package_data={ - 'pgwui_upload': [ + 'pgwui_upload_core': [ 'templates/*.mak', 'VERSION', ], @@ -186,7 +187,5 @@ setup( # # Setup an entry point to support PGWUI autoconfigure discovery. entry_points={ - 'pgwui.components': '.pgwui_upload = pgwui_upload', - 'pgwui.check_settings': - '.pgwui_upload = pgwui_upload.check_settings:check_settings'} + 'pgwui.components': '.pgwui_upload_core = pgwui_upload_core'} ) diff --git a/src/pgwui_upload/templates/upload.mak b/src/pgwui_upload/templates/upload.mak deleted file mode 100644 index 425c41f..0000000 --- a/src/pgwui_upload/templates/upload.mak +++ /dev/null @@ -1,116 +0,0 @@ -<%doc> - Copyright (C) 2015, 2018, 2020 The Meme Factory, Inc. - http://www.karlpinc.com/ - - This file is part of PGWUI_Upload. - - This program is free software: you can redistribute it and/or - modify it under the terms of the GNU Affero General Public License - as published by the Free Software Foundation, either version 3 of - the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public - License along with this program. If not, see - . - - Template for generic upload page. - - Karl O. Pinc - - This template uses the following variables in it's context: - - ask_about_literal_cols - - - - -<%! - from pgwui_common.path import asset_abspath - - auth_base_mak = asset_abspath('pgwui_common:templates/auth_base.mak') -%> - -<%inherit file="${auth_base_mak}" /> - -<%block name="title">${pgwui['pgwui_upload']['menu_label']} -<%block name="meta_keywords"> - - - -<%block name="meta_description"> - - - -<%block name="action_success"> -

Table (${table}) - successfully updated from a file containing ${lines} - lines! (Including column headings.) -

- - -

Upload File Into Database

- -<%def name="table_row(tab_index)"> - - - - - - - - - - -<%def name="trim_row(tab_index)"> - - - - - - - - - - -<% form_elements = [table_row, trim_row] %> - -% if ask_about_literal_cols: - <%def name="literal_row(tab_index)"> - - - - - - - - - - - <% form_elements.append(literal_row) %> -% endif - -${parent.upload_form(form_elements)} diff --git a/src/pgwui_upload/VERSION b/src/pgwui_upload_core/VERSION similarity index 100% rename from src/pgwui_upload/VERSION rename to src/pgwui_upload_core/VERSION diff --git a/src/pgwui_upload/__init__.py b/src/pgwui_upload_core/__init__.py similarity index 93% rename from src/pgwui_upload/__init__.py rename to src/pgwui_upload_core/__init__.py index 4a95019..1a08344 100644 --- a/src/pgwui_upload/__init__.py +++ b/src/pgwui_upload_core/__init__.py @@ -19,4 +19,4 @@ # Karl O. Pinc -from .pgwui_upload import includeme # noqa: F401 +from .pgwui_upload_core import includeme # noqa: F401 diff --git a/src/pgwui_upload/check_settings.py b/src/pgwui_upload_core/check_settings.py similarity index 72% rename from src/pgwui_upload/check_settings.py rename to src/pgwui_upload_core/check_settings.py index 1b725dd..542eb49 100644 --- a/src/pgwui_upload/check_settings.py +++ b/src/pgwui_upload_core/check_settings.py @@ -1,6 +1,6 @@ # Copyright (C) 2020 The Meme Factory, Inc. http://www.karlpinc.com/ -# This file is part of PGWUI_Upload. +# This file is part of PGWUI_Upload_Core. # # This program is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License @@ -20,10 +20,9 @@ # Karl O. Pinc from pgwui_common import checkset -from . import exceptions as upload_ex +from . import exceptions as upload_core_ex -PGWUI_COMPONENT = 'pgwui_upload' UPLOAD_SETTINGS = ['menu_label', 'literal_column_headings', ] @@ -31,17 +30,20 @@ REQUIRED_SETTINGS = [] BOOLEAN_SETTINGS = [] -def validate_literal_column_headings(errors, settings): +def validate_literal_column_headings(component, errors, settings): '''Make sure the values are those allowed ''' value = settings.get('literal_column_headings') if value is None: return if value not in ('on', 'off', 'ask'): - errors.append(upload_ex.BadLiteralColumnHeadingsError(value)) + errors.append( + upload_core_ex.BadLiteralColumnHeadingsError(component, value)) -def check_settings(component_config): +def check_settings( + component, all_setngs, required_setngs, boolean_setngs, + component_config): '''Check that all pgwui_upload specific settings are good. This includes: checking for unknown settings @@ -51,11 +53,11 @@ def check_settings(component_config): ''' errors = [] errors.extend(checkset.unknown_settings( - PGWUI_COMPONENT, UPLOAD_SETTINGS, component_config)) + component, all_setngs, component_config)) errors.extend(checkset.require_settings( - PGWUI_COMPONENT, REQUIRED_SETTINGS, component_config)) + component, required_setngs, component_config)) errors.extend(checkset.boolean_settings( - PGWUI_COMPONENT, BOOLEAN_SETTINGS, component_config)) - validate_literal_column_headings(errors, component_config) + component, boolean_setngs, component_config)) + validate_literal_column_headings(component, errors, component_config) return errors diff --git a/src/pgwui_upload/exceptions.py b/src/pgwui_upload_core/exceptions.py similarity index 85% rename from src/pgwui_upload/exceptions.py rename to src/pgwui_upload_core/exceptions.py index ac017b4..da590d7 100644 --- a/src/pgwui_upload/exceptions.py +++ b/src/pgwui_upload_core/exceptions.py @@ -19,32 +19,32 @@ # Karl O. Pinc -from pgwui_common import exceptions as common_ex from pgwui_core import exceptions as core_ex # PGWUI setting related exceptions -class UploadError(common_ex.Error): +class UploadError(core_ex.Error): pass class BadLiteralColumnHeadingsError(UploadError): - def __init__(self, value): + def __init__(self, component, value): super().__init__( - 'The "pgwui.pgwui_upload.literal_column_headings" PGWUI setting ' - ' must be "on", "off", "ask", or not present') + f'The "pgwui.{component}.literal_column_headings" PGWUI setting ' + f' is ({value}), it must be "on", "off", "ask", or the' + ' entire setting be omitted') # Upload related exceptions -class NoTableError(core_ex.PGWUIError): +class NoTableError(UploadError): '''No table uploaded''' def __init__(self, e, descr='', detail=''): super(NoTableError, self).__init__(e, descr, detail) -class BadTableError(core_ex.PGWUIError): +class BadTableError(UploadError): '''Supplied name does not work for a table or view''' def __init__(self, e, descr='', detail=''): super(BadTableError, self).__init__(e, descr, detail) @@ -68,7 +68,7 @@ class CannotInsertError(BadTableError): super(CannotInsertError, self).__init__(e, descr, detail) -class BadHeadersError(core_ex.PGWUIError): +class BadHeadersError(UploadError): '''The headers in the uploaded file are bad.''' def __init__(self, e, descr='', detail=''): super(BadHeadersError, self).__init__(e, descr, detail) diff --git a/src/pgwui_upload/pgwui_upload.py b/src/pgwui_upload_core/pgwui_upload_core.py similarity index 55% rename from src/pgwui_upload/pgwui_upload.py rename to src/pgwui_upload_core/pgwui_upload_core.py index b5fb682..05b48d1 100644 --- a/src/pgwui_upload/pgwui_upload.py +++ b/src/pgwui_upload_core/pgwui_upload_core.py @@ -1,6 +1,6 @@ # Copyright (C) 2018, 2020 The Meme Factory, Inc. http://www.karlpinc.com/ -# This file is part of PGWUI_Upload. +# This file is part of Pgwui_Upload_Core. # # This program is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License @@ -19,26 +19,12 @@ # Karl O. Pinc -'''Provide a way to configure PGWUI. +'''Provide a way to configure this PGWUI component. ''' -PGWUI_COMPONENT = 'pgwui_upload' -DEFAULT_UPLOAD_ROUTE = '/upload' -DEFAULT_UPLOAD_MENU_LABEL = 'upload -- Upload File Into Database' - - -def init_menu(config): - '''Add default menu information into settings when they are not present - ''' - settings = config.get_settings() - pgwui = settings.setdefault('pgwui', dict()) - pgwui.setdefault(PGWUI_COMPONENT, dict()) - pgwui[PGWUI_COMPONENT].setdefault( - 'menu_label', DEFAULT_UPLOAD_MENU_LABEL) +PGWUI_COMPONENT = 'pgwui_upload_core' def includeme(config): - '''Pyramid configuration for PGWUI_Upload + '''Pyramid configuration for Pgwui_Upload_Core ''' - init_menu(config) - config.add_route(PGWUI_COMPONENT, DEFAULT_UPLOAD_ROUTE) config.scan() diff --git a/src/pgwui_upload_core/templates/upload.mak b/src/pgwui_upload_core/templates/upload.mak new file mode 100644 index 0000000..7d6c097 --- /dev/null +++ b/src/pgwui_upload_core/templates/upload.mak @@ -0,0 +1,82 @@ +<%doc> + Copyright (C) 2015, 2018, 2020 The Meme Factory, Inc. + http://www.karlpinc.com/ + + This file is part of PGWUI_Upload_Core. + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Affero General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public + License along with this program. If not, see + . + + Template for generic upload pages. + + Karl O. Pinc + + This template uses the following variables in it's context: + + ask_about_literal_cols + + It makes available: + + append_elements(elementlist) Adds form elements to the end of the list + + + + +<%! + from pgwui_common.path import asset_abspath + + auth_base_mak = asset_abspath('pgwui_common:templates/auth_base.mak') +%> + +<%inherit file="${auth_base_mak}" /> + +<%def name="trim_row(tab_index)"> + + + + + + + + + + +<%def name="append_elements(form_elements)"> + <% form_elements.append(trim_row) %> + % if ask_about_literal_cols: + <%def name="literal_row(tab_index)"> + + + + + + + + + + + <% form_elements.append(literal_row) %> + % endif + diff --git a/src/pgwui_upload/views/__init__.py b/src/pgwui_upload_core/views/__init__.py similarity index 100% rename from src/pgwui_upload/views/__init__.py rename to src/pgwui_upload_core/views/__init__.py diff --git a/src/pgwui_upload/views/upload.py b/src/pgwui_upload_core/views/upload.py similarity index 64% rename from src/pgwui_upload/views/upload.py rename to src/pgwui_upload_core/views/upload.py index 35c8f8c..73023a6 100644 --- a/src/pgwui_upload/views/upload.py +++ b/src/pgwui_upload_core/views/upload.py @@ -1,7 +1,7 @@ # Copyright (C) 2015, 2018, 2020 The Meme Factory, Inc. # http://www.karlpinc.com/ -# This file is part of PGWUI_Upload. +# This file is part of PGWUI_Upload_Core. # # This program is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License @@ -24,31 +24,20 @@ # All data is presented to the db as a string, which could result # in problems with type coercion. -# Write python 3 compatible code. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division - -from pyramid.view import view_config +import attr import logging import markupsafe import psycopg2.errorcodes from psycopg2 import ProgrammingError -from pgwui_common.view import auth_base_view from pgwui_core.core import ( - UploadEngine, DataLineProcessor, - UploadDoubleTableForm, TabularFileUploadHandler, UploadData, doublequote, - escape_eol, - is_checked, ) -from pgwui_upload import exceptions as upload_ex +from pgwui_upload_core import exceptions as upload_ex log = logging.getLogger(__name__) @@ -74,7 +63,8 @@ class SaveLine(DataLineProcessor): self.cur.execute(self.insert_stmt, udl.tuples) -class TableUploadHandler(TabularFileUploadHandler): +@attr.s +class BaseTableUploadHandler(TabularFileUploadHandler): ''' Attributes: request A pyramid request instance @@ -83,12 +73,7 @@ class TableUploadHandler(TabularFileUploadHandler): ue cur ''' - - def make_form(self): - ''' - Make the upload form needed by this handler. - ''' - return UploadDoubleTableForm(self) + ue = attr.ib(default=None) def get_data(self): ''' @@ -108,13 +93,7 @@ class TableUploadHandler(TabularFileUploadHandler): Returns: A list of PGWUIError instances ''' - uf = self.uf - errors = super(TableUploadHandler, self).val_input() - - qualified_table = uf['table'] - if qualified_table == '': - errors.append(upload_ex.NoTableError( - 'No table or view name supplied')) + errors = super().val_input() self.double_validator(errors) @@ -122,7 +101,7 @@ class TableUploadHandler(TabularFileUploadHandler): def write(self, result, errors): '''Add double upload key into form.''' - response = super(TableUploadHandler, self).write(result, errors) + response = super().write(result, errors) self.write_double_key(response) return response @@ -180,29 +159,11 @@ class TableUploadHandler(TabularFileUploadHandler): else: return False - def factory(self, ue): - '''Make a db loader function from an UploadEngine. - - Input: - - Side Effects: - Yes, lots. + def validate_table(self, qualified_table): + '''Return schema and table names, or raise an exception + if the relation is not writable ''' - - self.ue = ue - self.cur = ue.cur - data = ue.data - qualified_table = self.uf['table'] - - quotecols = self.quote_columns() - if quotecols: - column_quoter = doublequote - else: - def column_quoter(x): - return x - schema, table = self.resolve_table(qualified_table) - if not self.good_table(schema, table): raise upload_ex.CannotInsertError( 'Cannot insert into supplied table or view', @@ -211,6 +172,48 @@ class TableUploadHandler(TabularFileUploadHandler): ' or you do not have the necessary' ' permissions to the table or view').format( markupsafe.escape(qualified_table))) + return (schema, table) + + def report_bad_cols(self, qualified_table, bad_cols, quotecols): + if quotecols: + detail = ('

The following columns are not in the ({0})' + ' table, or the first line of your file is not' + ' in the expected format and your column names' + ' have merged (all the names appear in a single' + ' list item, below), or the supplied column names' + ' do not match' + " the character case of the table's columns," + ' or you do not have permission to access' + ' the columns:

    ') + else: + detail = ('

    The following columns are not in the ({0})' + ' table, or the first line of your file is not' + ' in the expected format and your column names' + ' have merged (all the names appear in a single' + ' list item, below), ' + ' or the table has column names containing' + ' upper case characters, or you do not have' + ' permission to access the columns:

      ') + detail = detail.format(markupsafe.escape(qualified_table)) + + for bad_col in bad_cols: + detail += '
    • {0}
      • '.format(markupsafe.escape(bad_col)) + detail += '
    ' + raise upload_ex.BadHeadersError( + 'Header line contains unknown column names', + detail=detail) + + def get_column_quoter(self, quotecols): + if quotecols: + return doublequote + else: + def column_quoter(x): + return x + return column_quoter + + def build_insert_stmt( + self, data, qualified_table, quotecols, column_quoter): + schema, table = self.validate_table(qualified_table) column_sql = ('SELECT 1 FROM information_schema.columns' ' WHERE columns.table_name = %s' @@ -236,60 +239,16 @@ class TableUploadHandler(TabularFileUploadHandler): col_sep = ', ' if bad_cols: - if quotecols: - detail = ('

    The following columns are not in the ({0})' - ' table, or the supplied column names do not match' - " the character case of the table's columns," - ' or you do not have permission to access' - ' the columns:

      ') - else: - detail = ('

      The following columns are not in the ({0})' - ' table, or the table has column names containing' - ' upper case characters, or you do not have' - ' permission to access the columns:

        ') - detail = detail.format(markupsafe.escape(qualified_table)) - - for bad_col in bad_cols: - detail += '
      • {0}
        • '.format(markupsafe.escape(bad_col)) - detail += '
      ' - raise upload_ex.BadHeadersError( - 'Header line contains unknown column names', - detail=detail) + self.report_bad_cols(qualified_table, bad_cols, quotecols) - insert_stmt += ') VALUES({0})'.format(value_string) + return insert_stmt + ') VALUES({0})'.format(value_string) - return SaveLine(ue, self, insert_stmt) - - -@view_config(route_name='pgwui_upload', - renderer='pgwui_upload:templates/upload.mak') -@auth_base_view -def upload_view(request): - - response = UploadEngine(TableUploadHandler(request)).run() + def factory(self, ue): + '''Make a db loader function from an UploadEngine. - settings = request.registry.settings - quoter_setting = settings['pgwui'].get('literal_column_headings') - response['ask_about_literal_cols'] = quoter_setting == 'ask' - response.setdefault('pgwui', dict()) - response['pgwui']['pgwui_upload'] = settings['pgwui']['pgwui_upload'] + Input: - if response['db_changed']: - if is_checked(response['csv_checked']): - upload_fmt = 'CSV' - else: - upload_fmt = 'TAB' - log.info('Successful upload: DB {db}: Table ({table}):' - ' File ({filename}): Lines {lines}:' - ' Format {format}: Upload Null {null}: Null Rep ({null_rep}):' - ' Trim {trim}: By user {user}' - .format(filename=response['filename'], - lines=response['lines'], - format=upload_fmt, - null=is_checked(response['upload_null']), - null_rep=escape_eol(response['null_rep']), - trim=is_checked(response['trim_upload']), - db=response['db'], - table=response['table'], - user=response['user'])) - return response + Side Effects: + Yes, lots. + ''' + raise NotImplementedError() diff --git a/tests/test_check_settings.py b/tests/test_check_settings.py index 3833fd0..a101d37 100644 --- a/tests/test_check_settings.py +++ b/tests/test_check_settings.py @@ -1,6 +1,6 @@ # Copyright (C) 2020 The Meme Factory, Inc. http://www.karlpinc.com/ -# This file is part of PGWUI_Upload. +# This file is part of PGWUI_Upload_Core. # # This program is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License @@ -21,26 +21,16 @@ import pytest -import pgwui_upload.check_settings as check_settings +import pgwui_upload_core.check_settings as check_settings from pgwui_common import checkset from pgwui_testing import testing -from pgwui_upload import exceptions as upload_ex +from pgwui_upload_core import exceptions as upload_ex -# Activiate our pytest plugin +# Activiate the PGWUI pytest plugins pytest_plugins = ("pgwui",) -# Module packaging test - -def test_check_setting_is_pgwui_check_settings( - pgwui_check_settings_entry_point): - '''Ensure that pgwui_upload has a pgwui.check_settings entry point - ''' - assert (pgwui_check_settings_entry_point('pgwui_upload.check_settings') - is True) - - # Mocks mock_unknown_settings = testing.make_mock_fixture( @@ -55,61 +45,32 @@ mock_boolean_settings = testing.make_mock_fixture( # validate_literal_column_headings() +@pytest.mark.parametrize( + ('settings', 'error_class'), [ + ({}, None), + ({'literal_column_headings': 'on'}, None), + ({'literal_column_headings': 'off'}, None), + ({'literal_column_headings': 'ask'}, None), + ({'literal_column_headings': 'bad'}, + upload_ex.BadLiteralColumnHeadingsError)]) @pytest.mark.unittest -def test_validate_literal_column_headings_nosetting(): +def test_validate_literal_column_headings(settings, error_class): '''No error is delivered when there's no setting''' errors = [] - check_settings.validate_literal_column_headings(errors, {}) - - assert errors == [] - - -@pytest.mark.unittest -def test_validate_literal_column_headings_on(): - '''No error is delivered when the setting is "on"''' - errors = [] - check_settings.validate_literal_column_headings( - errors, {'literal_column_headings': 'on'}) - - assert errors == [] - - -@pytest.mark.unittest -def test_validate_literal_column_headings_off(): - '''No error is delivered when the setting is "off"''' - errors = [] - check_settings.validate_literal_column_headings( - errors, {'literal_column_headings': 'off'}) - - assert errors == [] - - -@pytest.mark.unittest -def test_validate_literal_column_headings_ask(): - '''No error is delivered when the setting is "ask"''' - errors = [] - check_settings.validate_literal_column_headings( - errors, {'literal_column_headings': 'ask'}) - - assert errors == [] - - -@pytest.mark.unittest -def test_validate_literal_column_headings_bad(): - '''delivers an error when given a bad value''' - errors = [] - check_settings.validate_literal_column_headings( - errors, {'literal_column_headings': 'bad'}) + check_settings.validate_literal_column_headings(None, errors, settings) - assert errors - assert isinstance( - errors[0], upload_ex.BadLiteralColumnHeadingsError) + if error_class: + assert len(errors) == 1 + assert isinstance( + errors[0], error_class) + else: + assert errors == [] literal_err = 'literal column headings error' mock_validate_literal_column_headings = testing.make_mock_fixture( check_settings, 'validate_literal_column_headings', - wraps=lambda errors, *args: errors.append(literal_err)) + wraps=lambda component, errors, *args: errors.append(literal_err)) # check_settings() @@ -131,7 +92,7 @@ def test_check_settings(mock_unknown_settings, mock_require_settings.return_value = require_retval mock_boolean_settings.return_value = boolean_retval - result = check_settings.check_settings({}) + result = check_settings.check_settings(None, [], [], [], {}) mock_unknown_settings.assert_called_once mock_require_settings.assert_called_once diff --git a/tests/test_pgwui_upload.py b/tests/test_pgwui_upload.py deleted file mode 100644 index 5309dbe..0000000 --- a/tests/test_pgwui_upload.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (C) 2019, 2020 The Meme Factory, Inc. http://www.karlpinc.com/ - -# This file is part of PGWUI_Upload. -# -# This program is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of -# the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with this program. If not, see -# . -# - -# Karl O. Pinc - -import pytest -import pyramid.testing - -import pgwui_upload.pgwui_upload as pgwui_upload - -from pgwui_testing import testing - -# Activiate our pytest plugin -pytest_plugins = ("pgwui",) - - -# Module packaging test - -def test_pgwui_upload_is_pgwui_component(pgwui_component_entry_point): - '''Ensure that pgwui_upload is a pgwui.component entry point - ''' - assert pgwui_component_entry_point('pgwui_upload') is True - - -# init_menu() - -@pytest.mark.unittest -def test_init_menu_default(): - '''The settings get the module's default value when no settings exist - ''' - with pyramid.testing.testConfig() as config: - - pgwui_upload.init_menu(config) - - new_settings = config.get_settings() - assert new_settings['pgwui']['pgwui_upload']['menu_label'] \ - == pgwui_upload.DEFAULT_UPLOAD_MENU_LABEL - - -@pytest.mark.unittest -def test_init_menu_no_default(): - '''The settings keep their value when they exist - ''' - test_menu_label = 'test label' - - with pyramid.testing.testConfig() as config: - sample_settings = config.get_settings() - - sample_settings['pgwui'] = dict() - sample_settings['pgwui']['pgwui_upload'] = dict() - sample_settings['pgwui']['pgwui_upload']['menu_label'] \ - = test_menu_label - - pgwui_upload.init_menu(config) - - new_settings = config.get_settings() - assert new_settings['pgwui']['pgwui_upload']['menu_label'] \ - == test_menu_label - - -mock_init_menu = testing.make_mock_fixture(pgwui_upload, 'init_menu') - - -# includeme() - -mock_add_route = testing.instance_method_mock_fixture('add_route') -mock_scan = testing.instance_method_mock_fixture('scan') - - -@pytest.mark.unittest -def test_includeme(mock_init_menu, mock_add_route, mock_scan): - '''init_menu, add_route, and scan are all called - ''' - with pyramid.testing.testConfig() as config: - mocked_add_route = mock_add_route(config) - mocked_scan = mock_scan(config) - - pgwui_upload.includeme(config) - - mock_init_menu.assert_called_once() - mocked_add_route.assert_called_once() - mocked_scan.assert_called_once() diff --git a/tests/test_pgwui_upload_core.py b/tests/test_pgwui_upload_core.py new file mode 100644 index 0000000..9504d5b --- /dev/null +++ b/tests/test_pgwui_upload_core.py @@ -0,0 +1,55 @@ +# Copyright (C) 2019, 2020 The Meme Factory, Inc. http://www.karlpinc.com/ + +# This file is part of Pgwui_Upload_Core. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +import pytest +import pyramid.testing + +import pgwui_upload_core.pgwui_upload_core as pgwui_upload_core + +from pgwui_testing import testing + +# Activiate our pytest plugin +pytest_plugins = ("pgwui",) + + +# Module packaging test + +def test_pgwui_upload_core_is_pgwui_component(pgwui_component_entry_point): + '''Ensure that pgwui_upload_core is a pgwui.component entry point + ''' + assert pgwui_component_entry_point('pgwui_upload_core') is True + + +# includeme() + +mock_scan = testing.instance_method_mock_fixture('scan') + + +@pytest.mark.unittest +def test_includeme(mock_scan): + '''scan() is called + ''' + with pyramid.testing.testConfig() as config: + mocked_scan = mock_scan(config) + + pgwui_upload_core.includeme(config) + + mocked_scan.assert_called_once() diff --git a/tests/views/test_upload.py b/tests/views/test_upload.py index 886f4aa..9335121 100644 --- a/tests/views/test_upload.py +++ b/tests/views/test_upload.py @@ -1,7 +1,7 @@ # Copyright (C) 2018, 2019, 2020 The Meme Factory, Inc. # http://www.karlpinc.com/ -# This file is part of PGWUI_Upload. +# This file is part of PGWUI_Upload_Core. # # This program is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License @@ -20,18 +20,21 @@ # Karl O. Pinc -import logging +import markupsafe import pytest from pyramid.testing import DummyRequest -from pyramid.threadlocal import get_current_request, get_current_registry from pgwui_common.__init__ import includeme as pgwui_common_includeme from pgwui_core import constants -from pgwui_upload.__init__ import includeme as pgwui_upload_includeme -from pgwui_upload.views import upload +from pgwui_upload_core.__init__ import includeme as pgwui_upload_core_includeme +from pgwui_upload_core import exceptions as upload_ex +from pgwui_upload_core.views import upload +from pgwui_testing import testing # Activiate our pytest plugin pytest_plugins = ("pgwui",) +# Mark all tests with "unittest" +pytestmark = pytest.mark.unittest # Constants CHANGED_RESPONSE = { @@ -59,6 +62,10 @@ DEFAULT_URLS = {'pgwui_upload': '/upload', 'home_page': '/'} +mock_escape = testing.make_mock_fixture( + markupsafe, 'escape') + + # Helper classes class MockUploadEngine(): @@ -89,13 +96,13 @@ def isolate_upload_view(monkeypatch, pyramid_request_config): monkeypatch.setattr(upload, 'UploadEngine', upload_engine) monkeypatch.setattr( - upload, 'TableUploadHandler', MockTableUploadHandler) + upload, 'BaseTableUploadHandler', MockTableUploadHandler) settings = pyramid_request_config.get_settings() settings['pgwui'] = settings.get('pgwui', dict()) settings['pgwui'].update({'home_page': HOME_PAGE_SETTINGS}) pgwui_common_includeme(pyramid_request_config) - pgwui_upload_includeme(pyramid_request_config) + pgwui_upload_core_includeme(pyramid_request_config) settings['pgwui'].update({'urls': DEFAULT_URLS}) pyramid_request_config.add_settings(settings) @@ -104,17 +111,29 @@ def isolate_upload_view(monkeypatch, pyramid_request_config): # Tests -# TableUploadHandler() +# BaseTableUploadHandler() + +# BaseTableUploadHandler.__init__() + +mock_tuh_init = testing.instance_method_mock_fixture('__init__') + + +# BaseTableUploadHandler.resolve_table() + +mock_resolve_table = testing.instance_method_mock_fixture('resolve_table') + + +# BaseTableUploadHandler.good_table() + +mock_good_table = testing.instance_method_mock_fixture('good_table') + @pytest.fixture def neuter_tableuploadhandler(monkeypatch): - '''Make TableUploadHander have a mock parent and the given uploadform + '''Make TableUploadHander have the given uploadform ''' def run(uploadform, request): - monkeypatch.setattr( - upload, 'TabularFileUploadHandler', MockTableUploadHandler) - - uh = upload.TableUploadHandler(request) + uh = upload.BaseTableUploadHandler(request) monkeypatch.setattr(uh, 'uf', uploadform) return uh @@ -122,7 +141,7 @@ def neuter_tableuploadhandler(monkeypatch): return run -# TableUploadHandler.get_form_column_quoter() +# BaseTableUploadHandler.quote_columns() @pytest.fixture def get_quote_columns(neuter_tableuploadhandler): @@ -180,76 +199,58 @@ def test_tableuploadhandler_quote_columns_ask_off(get_quote_columns): assert result is False -# upload_view() - -@pytest.fixture -def return_log_tuples(isolate_upload_view, caplog): - '''Get result and the caplog.record_tuples from the upload_view() call''' - caplog.set_level(logging.DEBUG) +# BaseTableUploadHandler.validate_table() - def run(response): - isolate_upload_view(response) - result = upload.upload_view(get_current_request()) - del result['pgwui'] # Remove variables added by pgwui view decorators - - return (result, caplog.record_tuples) - - return run - - -def test_upload_view_db_not_changed(return_log_tuples): - '''When the db did not change nothing logs''' - response = UNCHANGED_RESPONSE - (result, log_tuples) = return_log_tuples(response) - assert result == response - assert log_tuples == [] - - -def test_upload_view_db_changed_csv(return_log_tuples): - '''When the db did change from CSV input something logs''' - response = CHANGED_RESPONSE - response['csv_checked'] = constants.CHECKED - (result, log_tuples) = return_log_tuples(response) - - assert result == response - assert ([tup[:2] for tup in log_tuples] - == [('pgwui_upload.views.upload', logging.INFO)]) - - -def test_upload_view_db_changed_no_csv(return_log_tuples): - '''When the db did change from not-CSV input something logs''' - response = CHANGED_RESPONSE - response['csv_checked'] = constants.UNCHECKED - (result, log_tuples) = return_log_tuples(response) +def test_validate_table_good( + mock_resolve_table, mock_good_table, mock_escape): + '''When the table is good, the results of resolve_table() are returned + ''' + expected = ('schema', 'table') - assert result == response - assert ([tup[:2] for tup in log_tuples] - == [('pgwui_upload.views.upload', logging.INFO)]) + request = DummyRequest() + uh = upload.BaseTableUploadHandler(request) + mocked_resolve_table = mock_resolve_table(uh) + mocked_good_table = mock_good_table(uh) + mocked_resolve_table.return_value = expected + mocked_good_table.return_value = True + result = uh.validate_table(None) -def test_upload_view_literal_cols_ask(isolate_upload_view): - '''When literal_column_headings == ask the respose should reflect this''' + assert result == expected - response = UNCHANGED_RESPONSE - isolate_upload_view(response) - settings = get_current_request().registry.settings - settings['pgwui'].update({'literal_column_headings': 'ask'}) +def test_validate_table_bad( + mock_resolve_table, mock_good_table, mock_escape): + '''When the table is not good, the right exception is raised + ''' + expected = ('schema', 'table') - result = upload.upload_view(get_current_request()) + request = DummyRequest() + uh = upload.BaseTableUploadHandler(request) + mocked_resolve_table = mock_resolve_table(uh) + mocked_good_table = mock_good_table(uh) - assert result['ask_about_literal_cols'] + mocked_resolve_table.return_value = expected + mocked_good_table.return_value = False + with pytest.raises(upload_ex.CannotInsertError): + uh.validate_table(None) + assert True -def test_upload_view_literal_cols_noask(isolate_upload_view): - '''When literal_column_headings != ask the respose should reflect this''' - response = UNCHANGED_RESPONSE - isolate_upload_view(response) +# BaseTableUploadHandler.report_bad_cols() - settings = get_current_registry().settings - settings['pgwui'].update({'literal_column_headings': 'no'}) +@pytest.mark.parametrize( + ('quotecols',), [ + (True,), + (False,)]) +def test_report_bad_cols(mock_escape, quotecols): + '''The expected exception is raised + ''' - result = upload.upload_view(get_current_request()) + request = DummyRequest() + uh = upload.BaseTableUploadHandler(request) + with pytest.raises(upload_ex.BadHeadersError): + uh.report_bad_cols(None, ['col1', 'col2'], quotecols) - assert not(result['ask_about_literal_cols']) + assert True diff --git a/tox.ini b/tox.ini index 6591a14..bd889a2 100644 --- a/tox.ini +++ b/tox.ini @@ -21,9 +21,9 @@ commands = python setup.py sdist twine check dist/* flake8 . - py.test -m unittest --cov=pgwui_upload tests/ - py.test -m 'not unittest' --cov=pgwui_upload tests/ - # coverage run --source src/pgwui_upload -m py.test + py.test -m unittest --cov=pgwui_upload_core tests/ + py.test -m 'not unittest' --cov=pgwui_upload_core tests/ + # coverage run --source src/pgwui_upload_core -m py.test # coverage report [flake8] -- 2.34.1